Modeltime Forecasting
options & settings
For scrollable output
install packages
# install.packages("modeltime")
Load libs
library(tidyverse)
library(tidymodels)
library(modeltime)
library(timetk)
library(lubridate)
Load data
read.csv("day.csv") %>% head()
bike_transactions_tbl <- bike_sharing_daily %>%
select(dteday, cnt) %>%
set_names(c("date","value"))
head(bike_transactions_tbl)
Visualizing loaded data
bike_transactions_tbl %>%
timetk::plot_time_series(date, value)
# timetk::plot_time_series(date, value, .interactive = FALSE)
Train Test
- Setting
assess = "3 months" tells the function to use the last 3-months of data as the testing set.
- Setting
cumulative = TRUE tells the sampling to use all of the prior data as the training set.
splits <- bike_transactions_tbl %>%
time_series_split(assess = "3 months", cumulative = TRUE)
Visulaizing Train test split
splits %>%
timetk::tk_time_series_cv_plan() %>%
timetk::plot_time_series_cv_plan(date, value)
Modeling
Automatic Models
Auto Arima
model_fit_arima <- modeltime::arima_reg() %>%
parsnip::set_engine("auto_arima") %>%
parsnip::fit(value ~ date, training(splits))
model_fit_arima
parsnip model object
Fit time: 5.8s
Series: outcome
ARIMA(0,1,3) with drift
Coefficients:
ma1 ma2 ma3 drift
-0.6106 -0.1868 -0.0673 9.3169
s.e. 0.0396 0.0466 0.0398 4.6225
sigma^2 estimated as 730568: log likelihood=-5227.22
AIC=10464.44 AICc=10464.53 BIC=10486.74
Prophet
model_fit_prophet <- prophet_reg() %>%
set_engine("prophet", yearly.seasonality = TRUE) %>%
fit(value ~ date, training(splits))
model_fit_prophet
parsnip model object
Fit time: 1.4s
PROPHET Model
- growth: 'linear'
- n.changepoints: 25
- changepoint.range: 0.8
- yearly.seasonality: 'auto'
- weekly.seasonality: 'auto'
- daily.seasonality: 'auto'
- seasonality.mode: 'additive'
- changepoint.prior.scale: 0.05
- seasonality.prior.scale: 10
- holidays.prior.scale: 10
- logistic_cap: NULL
- logistic_floor: NULL
- extra_regressors: 0
ML Models
Machine learning models are more complex than the automated models. This complexity typically requires a workflow (sometimes called a pipeline in other languages). The general process goes like this:
Create Preprocessing Recipe
Create Model Specifications
Use Workflow to combine Model Spec and Preprocessing, and Fit Model
Preprocessing Recipe
First, I’ll create a preprocessing recipe using recipe() and adding time series steps. The process uses the “date” column to create 45 new features that I’d like to model. These include time-series signature features and fourier series.
recipe_spec <- recipe(value ~ date, training(splits)) %>%
step_timeseries_signature(date) %>%
step_rm(contains("am.pm"), contains("hour"), contains("minute"),
contains("second"), contains("xts")) %>%
step_fourier(date, period = 365, K = 5) %>%
step_dummy(all_nominal())
recipe_spec %>%
prep() %>%
juice()
With a recipe in-hand, we can set up our machine learning pipelines.
Elastic Net / glmnet
model_spec_glmnet <- linear_reg(penalty = 0.01, mixture = 0.5) %>%
set_engine("glmnet")
model_spec_glmnet
Linear Regression Model Specification (regression)
Main Arguments:
penalty = 0.01
mixture = 0.5
Computational engine: glmnet
workflow_fit_glmnet <- workflow() %>%
add_model(model_spec_glmnet) %>%
add_recipe(recipe_spec %>% step_rm(date)) %>%
fit(training(splits))
workflow_fit_glmnet
== Workflow [trained] ==========================================================
Preprocessor: Recipe
Model: linear_reg()
-- Preprocessor ----------------------------------------------------------------
5 Recipe Steps
* step_timeseries_signature()
* step_rm()
* step_fourier()
* step_dummy()
* step_rm()
-- Model -----------------------------------------------------------------------
Call: glmnet::glmnet(x = as.matrix(x), y = y, family = "gaussian", alpha = ~0.5)
Df %Dev Lambda
1 0 0.00 2856.00
2 1 5.71 2602.00
3 2 11.22 2371.00
4 2 19.05 2160.00
5 3 26.52 1968.00
6 3 33.31 1793.00
7 3 39.30 1634.00
8 4 44.58 1489.00
9 4 49.32 1357.00
10 5 53.41 1236.00
11 5 56.95 1126.00
12 5 59.98 1026.00
13 5 62.58 935.10
14 5 64.81 852.00
15 5 66.71 776.30
16 5 68.34 707.30
17 5 69.72 644.50
18 5 70.91 587.20
19 5 71.91 535.10
20 5 72.77 487.50
21 6 73.52 444.20
22 6 74.19 404.80
23 6 74.77 368.80
24 6 75.25 336.00
25 6 75.65 306.20
26 6 75.99 279.00
27 7 76.35 254.20
28 7 76.68 231.60
29 8 76.99 211.00
30 8 77.26 192.30
31 10 77.51 175.20
32 10 77.74 159.60
33 10 77.93 145.50
34 10 78.09 132.50
35 11 78.24 120.80
36 12 78.36 110.00
37 13 78.47 100.30
38 14 78.58 91.36
39 14 78.66 83.24
40 15 78.75 75.85
41 15 78.81 69.11
42 15 78.87 62.97
43 17 78.93 57.37
44 18 78.99 52.28
45 18 79.05 47.63
46 18 79.09 43.40
...
and 42 more lines.
RandomForest
model_spec_rf <- rand_forest(trees = 500, min_n = 50) %>%
set_engine("randomForest")
workflow_fit_rf <- workflow() %>%
add_model(model_spec_rf) %>%
add_recipe(recipe_spec %>% step_rm(date)) %>%
fit(training(splits))
workflow_fit_rf
== Workflow [trained] ==========================================================
Preprocessor: Recipe
Model: rand_forest()
-- Preprocessor ----------------------------------------------------------------
5 Recipe Steps
* step_timeseries_signature()
* step_rm()
* step_fourier()
* step_dummy()
* step_rm()
-- Model -----------------------------------------------------------------------
Call:
randomForest(x = as.data.frame(x), y = y, ntree = ~500, nodesize = ~50)
Type of random forest: regression
Number of trees: 500
No. of variables tried at each split: 15
Mean of squared residuals: 701813.6
% Var explained: 80.94
New Hybrid Models
Prophet Boost
The Prophet Boost algorithm combines Prophet with XGBoost to get the best of both worlds (i.e. Prophet Automation + Machine Learning). The algorithm works by:
First modeling the univariate series using Prophet
Using regressors supplied via the preprocessing recipe (remember our recipe generated 45 new features), and regressing the Prophet Residuals with the XGBoost model
We can set the model up using a workflow just like with the machine learning algorithms.
model_spec_prophet_boost <- prophet_boost() %>%
set_engine("prophet_xgboost", yearly.seasonality = TRUE)
workflow_fit_prophet_boost <- workflow() %>%
add_model(model_spec_prophet_boost) %>%
add_recipe(recipe_spec) %>%
fit(training(splits))
Modeltime Table
model_table <- modeltime_table(
model_fit_arima,
model_fit_prophet,
workflow_fit_glmnet,
workflow_fit_rf,
workflow_fit_prophet_boost
)
model_table
# Modeltime Table
Claibration
calibration_table <- model_table %>%
modeltime_calibrate(testing(splits))
calibration_table
# Modeltime Table
Forecasting
calibration_table %>%
modeltime_forecast(actual_data = bike_transactions_tbl) %>%
plot_modeltime_forecast() %>%
layout(legend = list(x = -0.5, y = 1.0))
# for more legend position options check out https://plotly.com/r/legend/
P_results1 <- calibration_table %>%
modeltime_forecast(actual_data = bike_transactions_tbl) %>%
plot_modeltime_forecast() %>%
layout(legend = list(orientation = "h", y = -0.2))
Using '.calibration_data' to forecast.
Error in library(htmlwidgetsl) :
there is no package called ‘htmlwidgetsl’
htmlwidgets::saveWidget(P_results1, "forecasting_results.html")
Accuracy (Testing set)
calibration_table %>%
modeltime_accuracy() %>%
table_modeltime_accuracy(.interactive = FALSE)
| .model_id |
.model_desc |
.type |
mae |
mape |
mase |
smape |
rmse |
rsq |
| 1 |
ARIMA(0,1,3) WITH DRIFT |
Test |
2540.11 |
474.89 |
2.74 |
46.00 |
3188.09 |
0.39 |
| 2 |
PROPHET |
Test |
3052.52 |
513.91 |
3.30 |
51.51 |
3707.56 |
0.29 |
| 3 |
GLMNET |
Test |
1243.53 |
332.96 |
1.34 |
29.20 |
1693.58 |
0.48 |
| 4 |
RANDOMFOREST |
Test |
1335.74 |
334.77 |
1.44 |
30.60 |
1851.42 |
0.48 |
| 5 |
PROPHET W/ XGBOOST ERRORS |
Test |
2528.39 |
412.40 |
2.73 |
46.02 |
3139.75 |
0.07 |
Refit and Forecast Forward
calibration_table %>%
# Remove arima model with low accuracy
filter(.model_id !=1) %>%
# Refit & Forecast Forward
modeltime_refit(bike_transactions_tbl) %>%
modeltime_forecast(h = "12 months", actual_data = bike_transactions_tbl) %>%
plot_modeltime_forecast() %>%
layout(legend = list(orientation = "h", y = -0.2))
LS0tDQp0aXRsZTogIk1vZGVsdGltZSBGb3JlY2FzdGluZyByLWJsb2dnZXJzIg0Kb3V0cHV0OiANCiAgaHRtbF9ub3RlYm9vazoNCiAgICBoaWdobGlnaHQ6IHRhbmdvDQogICAgZGZfcHJpbnQ6IHBhZ2VkDQogICAgdG9jOiB0cnVlDQogICAgdG9jX2Zsb2F0OiANCiAgICAgIGNvbGxhcHNlZDogZmFsc2UNCiAgICAgIHNtb290aF9zY3JvbGw6IGZhbHNlDQogICAgbnVtYmVyX3NlY3Rpb25zOiB0cnVlDQogICAgdG9jX2RlcHRoOiA2DQotLS0NCg0KDQojIE1vZGVsdGltZSBGb3JlY2FzdGluZw0KDQojIyBvcHRpb25zICYgc2V0dGluZ3MNCg0KRm9yIHNjcm9sbGFibGUgb3V0cHV0DQoNCmBgYHtjc3MsIGVjaG89RkFMU0V9DQouc2Nyb2xsLTEwMCB7DQogIG1heC1oZWlnaHQ6IDEwMHB4Ow0KICBvdmVyZmxvdy15OiBhdXRvOw0KICBiYWNrZ3JvdW5kLWNvbG9yOiBpbmhlcml0Ow0KfQ0KDQoNCmgxLCAjVE9DPnVsPmxpIHsNCiAgY29sb3I6ICNCNjREM0E7DQp9DQoNCmgyLCAjVE9DPnVsPnVsPmxpIHsNCiAgY29sb3I6ICMwMDAwMDA7DQp9DQoNCmgzLCAjVE9DPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICM2NDNjYjI7DQp9DQoNCmg0LCAjVE9DPnVsPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICNhZTAwNTg7DQp9DQoNCmg1LCAjVE9DPnVsPnVsPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICNmZmE0NDc7DQp9DQoNCmg2LCAjVE9DPnVsPnVsPnVsPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICNEQUUzRDk7DQp9DQoNCmBgYA0KDQoNCmBgYHtyIHNldHVwLCBpbmNsdWRlPUZBTFNFfQ0Ka25pdHI6Om9wdHNfY2h1bmskc2V0KGVjaG8gPSBUUlVFLCBtZXNzYWdlID0gRkFMU0UsIHdhcm5pbmcgPSBGQUxTRSwgZHBpID0gMzAwICxhdHRyLm91dHB1dD0nc3R5bGU9Im1heC1oZWlnaHQ6IDMwMHB4OyInLCBvdXQud2lkdGggPSAiMTAwJSIsIGZpZy53aWR0aD0xMikNCmBgYA0KDQoNCmBgYHtyfQ0Kb3B0aW9ucyhzY2lwZW4gPSA5OTkpDQpgYGANCg0KDQojIyBmcm9tDQoNCmh0dHBzOi8vd3d3LmJ1c2luZXNzLXNjaWVuY2UuaW8vY29kZS10b29scy8yMDIwLzA2LzI5L2ludHJvZHVjaW5nLW1vZGVsdGltZS5odG1sDQpodHRwczovL3d3dy5yLWJsb2dnZXJzLmNvbS8yMDIwLzA2L2ludHJvZHVjaW5nLW1vZGVsdGltZS10aWR5LXRpbWUtc2VyaWVzLWZvcmVjYXN0aW5nLXVzaW5nLXRpZHltb2RlbHMvDQoNCg0KKipiaWtlIHNoYXJpbmcgZGF0YSBmcm9tKioNCg0KQXZhaWxhYmxlIHdpdGhpbiBsaWJyYXJ5IGJ5IG5hbWUgYGJpa2Vfc2hhcmluZ19kYWlseWAgb3IgZG93bmxvYWQgZnJvbSBiZWxvdyBsaW5rDQoNCmh0dHBzOi8vYXJjaGl2ZS5pY3MudWNpLmVkdS9tbC9tYWNoaW5lLWxlYXJuaW5nLWRhdGFiYXNlcy8wMDI3NS8NCg0KKipSZWZlcmVuY2Ugb2Ygc29tZSBFREEgb24gQmlrZSBzaGFyaW5nIGRhdGEgKioNCg0KaHR0cHM6Ly93d3cucnB1YnMuY29tL3NoYXlpbmkvYmlrZV9zaGFyaW5nDQpodHRwOi8vcnN0dWRpby1wdWJzLXN0YXRpYy5zMy5hbWF6b25hd3MuY29tLzE1ODU5NV8xZjUyMGZkOGQ4ZTM0YTVhYjNhMTI3Mzc2ZjJmNjE2OS5odG1sDQoNCg0KIyMgaW5zdGFsbCBwYWNrYWdlcw0KDQpgYGB7ciB9DQojIGluc3RhbGwucGFja2FnZXMoIm1vZGVsdGltZSIpDQpgYGANCg0KIyMgTG9hZCBsaWJzDQoNCmBgYHtyIH0NCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeSh0aWR5bW9kZWxzKQ0KbGlicmFyeShtb2RlbHRpbWUpDQpsaWJyYXJ5KHRpbWV0aykNCmxpYnJhcnkobHVicmlkYXRlKQ0KYGBgDQoNCiMjIExvYWQgZGF0YQ0KDQpgYGB7cn0NCnJlYWQuY3N2KCJkYXkuY3N2IikgJT4lICBoZWFkKCkNCmBgYA0KDQoNCmBgYHtyfQ0KaGVhZChiaWtlX3NoYXJpbmdfZGFpbHkpDQpgYGANCg0KDQpgYGB7cn0NCmJpa2VfdHJhbnNhY3Rpb25zX3RibCA8LSBiaWtlX3NoYXJpbmdfZGFpbHkgJT4lIA0KICAgICAgICAgICAgICAgICAgICAgICAgICBzZWxlY3QoZHRlZGF5LCBjbnQpICU+JSANCiAgICAgICAgICAgICAgICAgICAgICAgICAgc2V0X25hbWVzKGMoImRhdGUiLCJ2YWx1ZSIpKQ0KDQpoZWFkKGJpa2VfdHJhbnNhY3Rpb25zX3RibCkNCmBgYA0KDQoNCiMjIyBWaXN1YWxpemluZyBsb2FkZWQgZGF0YQ0KDQpgYGB7cn0NCmJpa2VfdHJhbnNhY3Rpb25zX3RibCAlPiUgDQogIHRpbWV0azo6cGxvdF90aW1lX3NlcmllcyhkYXRlLCB2YWx1ZSkNCg0KIyAgdGltZXRrOjpwbG90X3RpbWVfc2VyaWVzKGRhdGUsIHZhbHVlLCAuaW50ZXJhY3RpdmUgPSBGQUxTRSkNCmBgYA0KDQojIyBUcmFpbiBUZXN0DQoNCg0KICAtIFNldHRpbmcgYGFzc2VzcyA9ICIzIG1vbnRocyJgIHRlbGxzIHRoZSBmdW5jdGlvbiB0byB1c2UgdGhlIGxhc3QgMy1tb250aHMgb2YgZGF0YSBhcyB0aGUgdGVzdGluZyBzZXQuDQogIC0gU2V0dGluZyBgY3VtdWxhdGl2ZSA9IFRSVUVgIHRlbGxzIHRoZSBzYW1wbGluZyB0byB1c2UgYWxsIG9mIHRoZSBwcmlvciBkYXRhIGFzIHRoZSB0cmFpbmluZyBzZXQuDQoNCg0KYGBge3J9DQpzcGxpdHMgPC0gYmlrZV90cmFuc2FjdGlvbnNfdGJsICU+JSANCiAgICAgICAgICAgIHRpbWVfc2VyaWVzX3NwbGl0KGFzc2VzcyA9ICIzIG1vbnRocyIsIGN1bXVsYXRpdmUgPSBUUlVFKQ0KYGBgDQoNCiMjIyBWaXN1bGFpemluZyBUcmFpbiB0ZXN0IHNwbGl0DQoNCmBgYHtyfQ0KDQpzcGxpdHMgJT4lIA0KICB0aW1ldGs6OnRrX3RpbWVfc2VyaWVzX2N2X3BsYW4oKSAlPiUgDQogIHRpbWV0azo6cGxvdF90aW1lX3Nlcmllc19jdl9wbGFuKGRhdGUsIHZhbHVlKQ0KYGBgDQoNCiMjIE1vZGVsaW5nDQoNCiMjIyBBdXRvbWF0aWMgTW9kZWxzDQoNCiMjIyMgQXV0byBBcmltYQ0KDQpgYGB7cn0NCm1vZGVsX2ZpdF9hcmltYSA8LSBtb2RlbHRpbWU6OmFyaW1hX3JlZygpICU+JSANCiAgICAgICAgICAgICAgICAgICAgcGFyc25pcDo6c2V0X2VuZ2luZSgiYXV0b19hcmltYSIpICU+JSANCiAgICAgICAgICAgICAgICAgICAgcGFyc25pcDo6Zml0KHZhbHVlIH4gZGF0ZSwgdHJhaW5pbmcoc3BsaXRzKSkNCg0KbW9kZWxfZml0X2FyaW1hDQpgYGANCg0KIyMjIyBQcm9waGV0DQoNCmBgYHtyfQ0KbW9kZWxfZml0X3Byb3BoZXQgPC0gcHJvcGhldF9yZWcoKSAlPiUNCiAgICAgICAgICAgICAgICAgICAgICBzZXRfZW5naW5lKCJwcm9waGV0IiwgeWVhcmx5LnNlYXNvbmFsaXR5ID0gVFJVRSkgJT4lDQogICAgICAgICAgICAgICAgICAgICAgZml0KHZhbHVlIH4gZGF0ZSwgdHJhaW5pbmcoc3BsaXRzKSkNCg0KbW9kZWxfZml0X3Byb3BoZXQNCmBgYA0KDQojIyMgTUwgTW9kZWxzDQoNCg0KTWFjaGluZSBsZWFybmluZyBtb2RlbHMgYXJlIG1vcmUgY29tcGxleCB0aGFuIHRoZSBhdXRvbWF0ZWQgbW9kZWxzLiBUaGlzIGNvbXBsZXhpdHkgdHlwaWNhbGx5IHJlcXVpcmVzIGEgKip3b3JrZmxvdyoqIChzb21ldGltZXMgY2FsbGVkIGEgcGlwZWxpbmUgaW4gb3RoZXIgbGFuZ3VhZ2VzKS4gVGhlIGdlbmVyYWwgcHJvY2VzcyBnb2VzIGxpa2UgdGhpczoNCg0KICAtICoqQ3JlYXRlIFByZXByb2Nlc3NpbmcgUmVjaXBlKioNCiAgDQogIC0gKipDcmVhdGUgTW9kZWwgU3BlY2lmaWNhdGlvbnMqKg0KICANCiAgLSAqKlVzZSBXb3JrZmxvdyB0byBjb21iaW5lIE1vZGVsIFNwZWMgYW5kIFByZXByb2Nlc3NpbmcsIGFuZCBGaXQgTW9kZWwqKg0KDQoNCiMjIyMgUHJlcHJvY2Vzc2luZyBSZWNpcGUNCg0KRmlyc3QsIEnigJlsbCBjcmVhdGUgYSBwcmVwcm9jZXNzaW5nIHJlY2lwZSB1c2luZyBgcmVjaXBlKClgIGFuZCBhZGRpbmcgdGltZSBzZXJpZXMgc3RlcHMuIFRoZSBwcm9jZXNzIHVzZXMgdGhlIGDigJxkYXRl4oCdYCBjb2x1bW4gdG8gY3JlYXRlIGA0NSBuZXcgZmVhdHVyZXNgIHRoYXQgSeKAmWQgbGlrZSB0byBtb2RlbC4gVGhlc2UgaW5jbHVkZSB0aW1lLXNlcmllcyBzaWduYXR1cmUgZmVhdHVyZXMgYW5kIGZvdXJpZXIgc2VyaWVzLg0KDQpgYGB7cn0NCnJlY2lwZV9zcGVjIDwtIHJlY2lwZSh2YWx1ZSB+IGRhdGUsIHRyYWluaW5nKHNwbGl0cykpICU+JSANCiAgICAgICAgICAgICAgICBzdGVwX3RpbWVzZXJpZXNfc2lnbmF0dXJlKGRhdGUpICU+JSANCiAgICAgICAgICAgICAgICBzdGVwX3JtKGNvbnRhaW5zKCJhbS5wbSIpLCBjb250YWlucygiaG91ciIpLCBjb250YWlucygibWludXRlIiksDQogICAgICAgICAgICAgICAgICAgICAgICBjb250YWlucygic2Vjb25kIiksIGNvbnRhaW5zKCJ4dHMiKSkgJT4lIA0KICAgICAgICAgICAgICAgIHN0ZXBfZm91cmllcihkYXRlLCBwZXJpb2QgPSAzNjUsIEsgPSA1KSAlPiUgDQogICAgICAgICAgICAgICAgc3RlcF9kdW1teShhbGxfbm9taW5hbCgpKQ0KDQpyZWNpcGVfc3BlYyAlPiUgDQogIHByZXAoKSAlPiUgDQogIGp1aWNlKCkNCmBgYA0KDQoNCldpdGggYSByZWNpcGUgaW4taGFuZCwgd2UgY2FuIHNldCB1cCBvdXIgbWFjaGluZSBsZWFybmluZyBwaXBlbGluZXMuDQoNCg0KIyMjIyBFbGFzdGljIE5ldCAvIGdsbW5ldA0KDQpgYGB7cn0NCm1vZGVsX3NwZWNfZ2xtbmV0IDwtIGxpbmVhcl9yZWcocGVuYWx0eSA9IDAuMDEsIG1peHR1cmUgPSAwLjUpICU+JSANCiAgICAgICAgICAgICAgICAgICAgICBzZXRfZW5naW5lKCJnbG1uZXQiKQ0KDQptb2RlbF9zcGVjX2dsbW5ldA0KYGBgDQoNCmBgYHtyfQ0Kd29ya2Zsb3dfZml0X2dsbW5ldCA8LSB3b3JrZmxvdygpICU+JSANCiAgYWRkX21vZGVsKG1vZGVsX3NwZWNfZ2xtbmV0KSAlPiUgDQogIGFkZF9yZWNpcGUocmVjaXBlX3NwZWMgJT4lIHN0ZXBfcm0oZGF0ZSkpICU+JSANCiAgZml0KHRyYWluaW5nKHNwbGl0cykpDQoNCndvcmtmbG93X2ZpdF9nbG1uZXQNCmBgYA0KDQojIyMjIFJhbmRvbUZvcmVzdA0KDQpgYGB7cn0NCm1vZGVsX3NwZWNfcmYgPC0gcmFuZF9mb3Jlc3QodHJlZXMgPSA1MDAsIG1pbl9uID0gNTApICU+JSANCiAgICAgICAgICAgICAgICAgIHNldF9lbmdpbmUoInJhbmRvbUZvcmVzdCIpDQoNCndvcmtmbG93X2ZpdF9yZiA8LSB3b3JrZmxvdygpICU+JSANCiAgYWRkX21vZGVsKG1vZGVsX3NwZWNfcmYpICU+JSANCiAgYWRkX3JlY2lwZShyZWNpcGVfc3BlYyAlPiUgc3RlcF9ybShkYXRlKSkgJT4lDQogIGZpdCh0cmFpbmluZyhzcGxpdHMpKQ0KDQp3b3JrZmxvd19maXRfcmYNCmBgYA0KDQojIyMgTmV3IEh5YnJpZCBNb2RlbHMNCg0KIyMjIyBQcm9waGV0IEJvb3N0DQoNClRoZSBQcm9waGV0IEJvb3N0IGFsZ29yaXRobSBjb21iaW5lcyBQcm9waGV0IHdpdGggWEdCb29zdCB0byBnZXQgdGhlIGJlc3Qgb2YgYm90aCB3b3JsZHMgKGkuZS4gUHJvcGhldCBBdXRvbWF0aW9uICsgTWFjaGluZSBMZWFybmluZykuIFRoZSBhbGdvcml0aG0gd29ya3MgYnk6DQoNCiAgMS4gRmlyc3QgbW9kZWxpbmcgdGhlIHVuaXZhcmlhdGUgc2VyaWVzIHVzaW5nIFByb3BoZXQNCiAgDQogIDIuIFVzaW5nIHJlZ3Jlc3NvcnMgc3VwcGxpZWQgdmlhIHRoZSBwcmVwcm9jZXNzaW5nIHJlY2lwZSAocmVtZW1iZXIgb3VyIHJlY2lwZSBnZW5lcmF0ZWQgNDUgbmV3IGZlYXR1cmVzKSwgYW5kIHJlZ3Jlc3NpbmcgdGhlIFByb3BoZXQgUmVzaWR1YWxzIHdpdGggdGhlIFhHQm9vc3QgbW9kZWwNCg0KV2UgY2FuIHNldCB0aGUgbW9kZWwgdXAgdXNpbmcgYSB3b3JrZmxvdyBqdXN0IGxpa2Ugd2l0aCB0aGUgbWFjaGluZSBsZWFybmluZyBhbGdvcml0aG1zLg0KDQpgYGB7cn0NCm1vZGVsX3NwZWNfcHJvcGhldF9ib29zdCA8LSBwcm9waGV0X2Jvb3N0KCkgJT4lIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2V0X2VuZ2luZSgicHJvcGhldF94Z2Jvb3N0IiwgeWVhcmx5LnNlYXNvbmFsaXR5ID0gVFJVRSkNCg0Kd29ya2Zsb3dfZml0X3Byb3BoZXRfYm9vc3QgPC0gd29ya2Zsb3coKSAlPiUgDQogIGFkZF9tb2RlbChtb2RlbF9zcGVjX3Byb3BoZXRfYm9vc3QpICU+JSANCiAgYWRkX3JlY2lwZShyZWNpcGVfc3BlYykgJT4lIA0KICBmaXQodHJhaW5pbmcoc3BsaXRzKSkNCg0KDQpgYGANCg0KDQojIyBNb2RlbHRpbWUgVGFibGUNCg0KYGBge3J9DQptb2RlbF90YWJsZSA8LSBtb2RlbHRpbWVfdGFibGUoDQogICAgICAgICAgICAgICAgICBtb2RlbF9maXRfYXJpbWEsDQogICAgICAgICAgICAgICAgICBtb2RlbF9maXRfcHJvcGhldCwNCiAgICAgICAgICAgICAgICAgIHdvcmtmbG93X2ZpdF9nbG1uZXQsDQogICAgICAgICAgICAgICAgICB3b3JrZmxvd19maXRfcmYsDQogICAgICAgICAgICAgICAgICB3b3JrZmxvd19maXRfcHJvcGhldF9ib29zdA0KICAgICAgICAgICAgICAgICAgKQ0KDQptb2RlbF90YWJsZQ0KYGBgDQoNCg0KIyMgQ2xhaWJyYXRpb24NCg0KYGBge3J9DQpjYWxpYnJhdGlvbl90YWJsZSA8LSBtb2RlbF90YWJsZSAlPiUgDQogIG1vZGVsdGltZV9jYWxpYnJhdGUodGVzdGluZyhzcGxpdHMpKQ0KDQpjYWxpYnJhdGlvbl90YWJsZQ0KYGBgDQoNCiMjIEZvcmVjYXN0aW5nDQoNCmBgYHtyfQ0KbGlicmFyeShwbG90bHkpDQpgYGANCg0KDQpgYGB7cn0NCmNhbGlicmF0aW9uX3RhYmxlICU+JSANCiAgbW9kZWx0aW1lX2ZvcmVjYXN0KGFjdHVhbF9kYXRhID0gYmlrZV90cmFuc2FjdGlvbnNfdGJsKSAlPiUgDQogIHBsb3RfbW9kZWx0aW1lX2ZvcmVjYXN0KCkgJT4lIA0KICBsYXlvdXQobGVnZW5kID0gbGlzdCh4ID0gLTAuNSwgeSA9IDEuMCkpDQoNCiAgIyBmb3IgbW9yZSBsZWdlbmQgcG9zaXRpb24gb3B0aW9ucyBjaGVjayBvdXQgaHR0cHM6Ly9wbG90bHkuY29tL3IvbGVnZW5kLw0KDQpgYGANCg0KDQpgYGB7cn0NClBfcmVzdWx0czEgPC0gY2FsaWJyYXRpb25fdGFibGUgJT4lIA0KICBtb2RlbHRpbWVfZm9yZWNhc3QoYWN0dWFsX2RhdGEgPSBiaWtlX3RyYW5zYWN0aW9uc190YmwpICU+JSANCiAgcGxvdF9tb2RlbHRpbWVfZm9yZWNhc3QoKSAlPiUgDQogIGxheW91dChsZWdlbmQgPSBsaXN0KG9yaWVudGF0aW9uID0gImgiLCB5ID0gLTAuMikpDQoNClBfcmVzdWx0czENCmBgYA0KDQpgYGB7cn0NCmxpYnJhcnkoaHRtbHdpZGdldHMpDQpgYGANCg0KYGBge3J9DQpodG1sd2lkZ2V0czo6c2F2ZVdpZGdldChQX3Jlc3VsdHMxLCAiZm9yZWNhc3RpbmdfcmVzdWx0cy5odG1sIikNCmBgYA0KDQoNCiMjIEFjY3VyYWN5IChUZXN0aW5nIHNldCkNCg0KYGBge3J9DQpjYWxpYnJhdGlvbl90YWJsZSAlPiUgDQogIG1vZGVsdGltZV9hY2N1cmFjeSgpICU+JSANCiAgdGFibGVfbW9kZWx0aW1lX2FjY3VyYWN5KC5pbnRlcmFjdGl2ZSA9IEZBTFNFKQ0KYGBgDQoNCg0KIyMgUmVmaXQgYW5kIEZvcmVjYXN0IEZvcndhcmQNCg0KYGBge3IsIG91dC53aWR0aD0gIjEwMCUifQ0KY2FsaWJyYXRpb25fdGFibGUgJT4lIA0KICANCiAgIyBSZW1vdmUgYXJpbWEgbW9kZWwgd2l0aCBsb3cgYWNjdXJhY3kNCiAgZmlsdGVyKC5tb2RlbF9pZCAhPTEpICU+JSANCiAgDQogICMgUmVmaXQgJiBGb3JlY2FzdCBGb3J3YXJkDQogIG1vZGVsdGltZV9yZWZpdChiaWtlX3RyYW5zYWN0aW9uc190YmwpICU+JSANCiAgbW9kZWx0aW1lX2ZvcmVjYXN0KGggPSAiMTIgbW9udGhzIiwgYWN0dWFsX2RhdGEgPSBiaWtlX3RyYW5zYWN0aW9uc190YmwpICU+JQ0KICBwbG90X21vZGVsdGltZV9mb3JlY2FzdCgpICU+JSANCiAgbGF5b3V0KGxlZ2VuZCA9IGxpc3Qob3JpZW50YXRpb24gPSAiaCIsIHkgPSAtMC4yKSkNCmBgYA0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0K